package com.moneycollect.webspider.builtwith;

import com.alibaba.excel.EasyExcel;
import com.alibaba.excel.ExcelWriter;
import com.alibaba.excel.write.metadata.WriteSheet;
import com.fasterxml.jackson.core.type.TypeReference;
import com.moneycollect.webspider.atome.entity.BrandResponse;
import com.moneycollect.webspider.atome.entity.ContactEntity;
import com.moneycollect.webspider.atome.entity.ExportEntity;
import com.moneycollect.webspider.builtwith.entity.BuildWithExportEntity;
import com.moneycollect.webspider.builtwith.entity.SiteInfoEntity;
import com.moneycollect.webspider.common.*;
import org.apache.commons.lang3.StringUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.springframework.util.CollectionUtils;

import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

/**
 * @author xuhuiqian
 * @since 2021/12/6 12:03
 */
public class BuildWithSpider {
    private static final List<String> saasUrlList = List.of("https://trends.builtwith.com/websitelist/Shopify/Hong-Kong",
            "https://trends.builtwith.com/websitelist/Shopify-Plus/Hong-Kong",
            "https://trends.builtwith.com/websitelist/Magento/Hong-Kong",
            "https://trends.builtwith.com/websitelist/Magento-Enterprise/Hong-Kong",
            "https://trends.builtwith.com/websitelist/WooCommerce-Checkout/Hong-Kong",
            "https://trends.builtwith.com/websitelist/OpenCart/Hong-Kong",
            "https://trends.builtwith.com/websitelist/Wix-Stores/Hong-Kong",
            "https://trends.builtwith.com/websitelist/Shopline/Hong-Kong");

    public static void  run () {
        try {
            getSaasPlatformSites();
        } catch (Exception e) {
            e.printStackTrace();
        }

       /* try {
            getContact();
        } catch (Exception e) {
            e.printStackTrace();
        }*/

        /*try {
            getContactContent();
        } catch (Exception e) {
            e.printStackTrace();
        }*/

        /*try {
            // 去重导出
            filterResult();
        } catch (Exception e) {
            e.printStackTrace();
        }*/
    }

    // region 获取商家站点列表
    private static void getSaasPlatformSites() throws Exception {
        List<SiteInfoEntity> result = new ArrayList<>();
        for(var plat : saasUrlList) {
            var htmlContent = GetHtmlHelper.getHomePageContentByHttpClient(plat);

            if(StringUtils.isNotBlank(htmlContent)) {
                Document doc = Jsoup.parseBodyFragment(htmlContent);
                var rowElements = doc.getElementsByAttribute("data-domain");
                if(rowElements.size() > 0) {
                    for(var ele : rowElements) {
                        var tdElements = ele.select(".pl-0");
                        if(tdElements.size() > 0) {
                            var siteUrl = tdElements.get(0).text();
                            var entity = SiteInfoEntity.builder().webSiteUrl(siteUrl).build();
                            var platformName = getPlatformNameFromUrl(plat);
                            entity.setSaasPlatform(platformName);
                            result.add(entity);
                        }
                    }
                }
            }

        }

        var jsonResult = JsonUtil.toJson(result);
        FileHelper.saveFile(jsonResult, "E:\\build_with_list_2.txt");
    }

    private static String getPlatformNameFromUrl(String platformUrl) {
        var parts = platformUrl.split("/");
        return parts[parts.length -2];
    }

    // endregion

    // region 获取商家联系页面的URL
    private static void getContact() throws Exception {
        var merchantListJson = FileHelper.readAllFile4Utf8("E:\\build_with_list.txt");
        var brandList = JsonUtil.parseObject(merchantListJson, new TypeReference<List<SiteInfoEntity>>() {
        });

        if (!CollectionUtils.isEmpty(brandList)) {
            for (var b : brandList) {
                try {
                    if (StringUtils.isBlank(b.getWebSiteUrl()) || StringUtils.isNotBlank(b.getContactUsUrl())) {
                        continue;
                    }

                    String contactUsUrl = null;
                    var requestUrl = fixWebSiteUrl(b.getWebSiteUrl());

                    var htmlContent = GetHtmlHelper.getHomePageContentByHttpClient(requestUrl);
                    contactUsUrl = searchContactUsUrl(htmlContent);
                    if (StringUtils.isBlank(contactUsUrl)) {
                        htmlContent = GetHtmlHelper.getUrlHtmlContentByCdp(requestUrl);
                        contactUsUrl = searchContactUsUrl(htmlContent);
                    }

                    b.setContactUsUrl(contactUsUrl);
                    System.out.println("[" + b.getSaasPlatform() + "][" + b.getWebSiteUrl() + "]get contract url");
                } catch (Exception e) {
                    System.out.println("[" + b.getSaasPlatform() + "][" + b.getWebSiteUrl() + "]get contract url exception:" + e.getMessage());
                }
            }
        }

        var result = JsonUtil.toJson(brandList);
        FileHelper.saveFile(result, "E:\\build_with_list_contact_us.txt");
    }

    private static final Pattern contactUsPattern = Pattern.compile("(聯繫我們|联系我们|聯絡我們|联络我们|Contact Us|联络|Contact)+");
    private static String searchContactUsUrl(String htmlDoc) {
        if (StringUtils.isBlank(htmlDoc)) {
            return null;
        }
        try {
            Document doc = Jsoup.parseBodyFragment(htmlDoc);
            var allElement = doc.getElementsMatchingText(contactUsPattern);
            if (allElement.size() > 0) {
                for (org.jsoup.nodes.Element element : allElement) {
                    var href = element.attr("href");
                    if (StringUtils.isNotBlank(href)) {
                        return href;
                    }
                }
            }
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }

    private static String fixWebSiteUrl(String url) {
        if(StringUtils.isBlank(url)) {
            return  url;
        }

        if(url.startsWith("http")) {
            return url;
        }
        return "https://" + url;
    }

    // endregion

    // region 从联系页面分析联系方式
    private static void getContactContent() throws Exception {
        var bankJson = FileHelper.readAllFile4Utf8("E:\\build_with_list_contact_us.txt");
        var brandList = JsonUtil.parseObject(bankJson, new TypeReference<List<SiteInfoEntity>>() {
        });

        if (!CollectionUtils.isEmpty(brandList)) {
            for (var b : brandList) {
                if (StringUtils.isBlank(b.getContactUsUrl())) {
                    continue;
                }

                var fixUrl = fixContactUsUrl(b.getWebSiteUrl(), b.getContactUsUrl());
                if (StringUtils.isBlank(fixUrl) || !fixUrl.toLowerCase().startsWith("http")) {
                    continue;
                }

                extractContactContent(b, fixUrl, null);
                System.out.println("[" + b.getSaasPlatform() + "][" + b.getWebSiteUrl() + "]get contract content");
            }
        }

        var result = JsonUtil.toJson(brandList);
        FileHelper.saveFile(result, "E:\\build_with_list_contact.txt");
    }

    private static String fixContactUsUrl(String webSiteUrl, String contactUsUrl) {
        if (contactUsUrl.toLowerCase().startsWith("http")) {
            return contactUsUrl;
        }

        if (contactUsUrl.toLowerCase().startsWith("mailto")) {
            return null;
        }

        var root = StringUtils.stripEnd(webSiteUrl, "/");
        var relatedUrl = StringUtils.stripStart(contactUsUrl, "/");
        var resultUrl =  root + "/" + relatedUrl;
        if(!resultUrl.startsWith("http")) {
            resultUrl = "https://" + resultUrl;
        }

        return resultUrl;
    }

    private static void extractContactContent(SiteInfoEntity b, String url, String content) {
        ContactEntity contactUsEntity = null;
        if (StringUtils.isNotBlank(content)) {
            contactUsEntity = HandlerContact.getContactContent(content);
        } else if(StringUtils.isNotBlank(url)) {
            String htmlContent = GetHtmlHelper.getHomePageContentByHttpClient(url);
            contactUsEntity = HandlerContact.getContactContent(htmlContent);
            if (CollectionUtils.isEmpty(contactUsEntity.getEmail()) && CollectionUtils.isEmpty(contactUsEntity.getPhone())) {
                htmlContent = GetHtmlHelper.getUrlHtmlContentByCdp(url);
                contactUsEntity = HandlerContact.getContactContent(htmlContent);
            }
        }

        if (b.getEmail() == null) {
            b.setEmail(new ArrayList<>());
        }
        if (b.getPhone() == null) {
            b.setPhone(new ArrayList<>());
        }

        if(contactUsEntity != null && contactUsEntity.getEmail() != null) {
            b.getEmail().addAll(contactUsEntity.getEmail());
        }

        if(contactUsEntity != null && contactUsEntity.getPhone() != null){
            b.getPhone().addAll(contactUsEntity.getPhone());
        }
    }

    // endregion

    // region 去重导出
    private static void filterResult() throws Exception {
        var bankJson = FileHelper.readAllFile4Utf8("E:\\build_with_list_contact.txt");
        var brandList = JsonUtil.parseObject(bankJson, new TypeReference<List<SiteInfoEntity>>() {
        });
        if (!CollectionUtils.isEmpty(brandList)) {
            for (var b : brandList) {
                HandlerContact.filterEmail(b);
                HandlerContact.filterPhone(b);
            }
        }

        var result = JsonUtil.toJson(brandList);
        FileHelper.saveFile(result, "E:\\build_with_result.txt");

        exportData(brandList);
    }

    private static void exportData(List<SiteInfoEntity> brandList) {
        String path = "E:\\buildWith.xls";
        ExcelWriter excelWriter = EasyExcel.write(path, BuildWithExportEntity.class)
                .registerConverter(new EasyExcelListConvertor())
                .build();

        if (!CollectionUtils.isEmpty(brandList)) {
            writeSheet(brandList, excelWriter);
        }

        // 关闭
        excelWriter.finish();
        System.out.println("finish");
    }

    private static void writeSheet(List<SiteInfoEntity> brandList, ExcelWriter excelWriter) {
        var cc = Comparator.<SiteInfoEntity, Integer>comparing(p ->
        {
            if(CollectionUtils.isEmpty(p.getEmail()) && CollectionUtils.isEmpty(p.getPhone())) {
                return 1;
            }
            return 0;
        });

        for(var s : saasUrlList) {
            var platformName = getPlatformNameFromUrl(s);
            var sourceData = brandList.stream().filter(p -> p.getSaasPlatform().equals(platformName)).collect(Collectors.toList());
            sourceData = sourceData.stream().sorted(cc).collect(Collectors.toList());
            WriteSheet sheet = EasyExcel.writerSheet(platformName).build();

            List<BuildWithExportEntity> exportDataList = new ArrayList<>();
            for(var d : sourceData) {
                var exportEntity = BuildWithExportEntity.builder()
                        .webSiteUrl(d.getWebSiteUrl())
                        .saasPlatform(platformName)
                        .contactUsUrl(d.getContactUsUrl())
                        .email(d.getEmail())
                        .phone(d.getPhone())
                        .build();
                exportDataList.add(exportEntity);
            }

            excelWriter.write(exportDataList, sheet);
        }
    }

    // endregion
}
